package org.report.util;

import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;

/**
 * @author Administrator
 * @describe html页面解析抓取工具
 * @date 2024/9/12 0012 15:01
 */
public class HtmlContentGrabUtils {
    public static void main(String[] args) throws Exception {
        List<JSONObject> areaInfo = getAreaInfo("https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023");
        File fileSql = new File("D:\\kxz\\中国行政区域关系.sql");
        File fileJson = new File("D:\\kxz\\中国行政区域关系.json");
        if (!fileSql.exists()) {
            if (!fileSql.getParentFile().exists()) {
                fileSql.getParentFile().mkdir();
            }
            fileSql.createNewFile();
        }
        if (!fileJson.exists()) {
            if (!fileJson.getParentFile().exists()) {
                fileJson.getParentFile().mkdir();
            }
            fileJson.createNewFile();
        }
        FileWriter fileWriter = new FileWriter(fileJson);
        fileWriter.write(JSONObject.toJSONString(areaInfo));
        fileWriter.flush();
        fileWriter.close();
        fileWriter = new FileWriter(fileSql);
        String sql = "INSERT INTO `area`(`id`, `code`, `name`, `parent_code`, `level`) VALUES (%s, `%s`, `%s`, `%s`, %s)\n";
        int id = 1;
        List<String> sqlList = new ArrayList<>();
        generateAreaSql(sql, sqlList, areaInfo, id, "", 1);
        for (String s : sqlList) {
            fileWriter.write(s);
        }
        fileWriter.flush();
        fileWriter.close();
        System.out.println(JSONObject.toJSONString(areaInfo));
    }

    private static void generateAreaSql(String sql, List<String> sqlList, List<JSONObject> areaInfo, int id, String parentCode, int level) {
        for (JSONObject jsonObject : areaInfo) {
            String code = jsonObject.getString("code");
            String name = jsonObject.getString("name");
            sqlList.add(sql.formatted(id, code, name, parentCode, level));
            id++;
            JSONArray jsonArray = jsonObject.getJSONArray("children");
            if (!CollectionUtil.isEmpty(jsonArray)) {
                generateAreaSql(sql, sqlList, areaInfo, id, code, level + 1);
            }
        }
    }

    /**
     * 抓取国家统计局行政区域代码信息
     *
     * @return
     */
    public static List<JSONObject> getAreaInfo(String baseUrl) throws Exception {
        Random random = new Random();
        List<JSONObject> provinceList = new ArrayList<>();
        String provinceHtml = getHtml(baseUrl + "/index.html");
        Document doc = Jsoup.parse(provinceHtml);
        // 获取省信息
        Elements provinceElements = doc.getElementsByAttributeValue("class", "provincetr");
        if (provinceElements != null && provinceElements.size() > 0) {
            for (Element provinceElement : provinceElements) {
                for (Node childNode : provinceElement.childNodes()) {
                    Element tdNode = (Element) childNode;
                    Element aNode = (Element) tdNode.childNodes().get(0);
                    String href = aNode.attr("href");
                    TextNode textNode = (TextNode) aNode.childNodes().get(0);
                    JSONObject province = new JSONObject();
                    province.put("name", textNode.text());
                    province.put("code", href.replace(".html", ""));
                    System.out.println("开始抓取省级[" + province.getString("name") + "]");
                    Thread.sleep(500 + random.nextInt(200));
                    String cityUrl = baseUrl + "/" + href;
                    List<JSONObject> cityList = new ArrayList<>();
                    // 获取市级列表
                    String cityHtml = getHtml(cityUrl);
                    Document cityDoc = Jsoup.parse(cityHtml);
                    Elements cityElements = cityDoc.getElementsByAttributeValue("class", "citytr");
                    if (cityElements != null && cityElements.size() > 0) {
                        for (Element cityElement : cityElements) {
                            Element cityCodeANode = (Element) cityElement.childNode(0).childNode(0);
                            Element cityNameANode = (Element) cityElement.childNode(1).childNode(0);
                            JSONObject city = new JSONObject();
                            city.put("code", ((TextNode) cityCodeANode.childNode(0)).text());
                            city.put("name", ((TextNode) cityNameANode.childNode(0)).text());
                            System.out.println("开始抓取市级[" + province.getString("name") + "][" + city.getString("name") + "]");
                            Thread.sleep(500 + random.nextInt(200));
                            String countyUrl = baseUrl + "/" + cityNameANode.attr("href");
                            // 获取区(城市)
                            List<JSONObject> countyList = new ArrayList<>();
                            String countyHtml = getHtml(countyUrl);
                            Document countyDoc = Jsoup.parse(countyHtml);
                            Elements countyElements = countyDoc.getElementsByAttributeValue("class", "countytr");
                            if (countyElements != null && countyElements.size() > 0) {
                                for (Element countyElement : countyElements) {
                                    List<Node> countyNodes = countyElement.childNodes();
                                    // 获取街道
                                    List<JSONObject> townList = new ArrayList<>();
                                    JSONObject county = new JSONObject();
                                    Node countyNode0 = countyNodes.get(0).childNode(0);
                                    Node countyNode1 = countyNodes.get(1).childNode(0);
                                    if (countyNode0 instanceof TextNode) {
                                        county.put("code", ((TextNode) countyNode0).text());
                                        county.put("name", ((TextNode) countyNode1).text());
                                    } else {
                                        county.put("code", ((TextNode) countyNode0.childNode(0)).text());
                                        county.put("name", ((TextNode) countyNode1.childNode(0)).text());
                                    }
                                    System.out.println("开始抓取区级[" + province.getString("name") + "][" + city.getString("name") + "]" + county.getString("name") + "]");
                                    Thread.sleep(500 + random.nextInt(200));
                                    if (CollectionUtil.isEmpty(countyNode1.childNodes())) {
                                        continue;
                                    }
                                    String townUrl = baseUrl + "/" + province.get("code") + "/" + countyNode1.attr("href");
                                    String townHtml = getHtml(townUrl);
                                    Document townDoc = Jsoup.parse(townHtml);
                                    Elements townElements = townDoc.getElementsByAttributeValue("class", "towntr");
                                    if (townElements != null && townElements.size() > 0) {
                                        for (Element townElement : townElements) {
                                            List<Node> townNodes = townElement.childNodes();
                                            JSONObject town = new JSONObject();
                                            List<JSONObject> villageList = new ArrayList<>();
                                            Node townNode0 = townNodes.get(0).childNode(0);
                                            Node townNode1 = townNodes.get(1).childNode(0);
                                            if (townNode0 instanceof TextNode) {
                                                town.put("code", ((TextNode) townNode0).text());
                                                town.put("name", ((TextNode) townNode1).text());
                                            } else {
                                                town.put("code", ((TextNode) townNode0.childNode(0)).text());
                                                town.put("name", ((TextNode) townNode1.childNode(0)).text());
                                            }
                                            if (CollectionUtil.isEmpty(townNode1.childNodes())) {
                                                continue;
                                            }
                                            System.out.println("开始抓取街道级[" + province.getString("name") + "][" + city.getString("name") + "]" + county.getString("name") + "]" + town.getString("name") + "]");
                                            Thread.sleep(500 + random.nextInt(200));
                                            String code = town.getString("code");
                                            String villageUrl = baseUrl + "/" + code.substring(0, 2) + "/" + code.substring(2, 4) + "/" + townNode1.attr("href");
                                            // 获取社区
                                            String villageHtml = getHtml(villageUrl);
                                            Document villageDoc = Jsoup.parse(villageHtml);
                                            Elements villageElements = villageDoc.getElementsByAttributeValue("class", "villagetr");
                                            if (villageElements != null && villageElements.size() > 0) {
                                                for (Element villageElement : villageElements) {
                                                    JSONObject village = new JSONObject();
                                                    village.put("code", ((TextNode) villageElement.childNode(0).childNode(0)).text());
                                                    village.put("name", ((TextNode) villageElement.childNode(2).childNode(0)).text());
                                                    System.out.println("开始抓取社区级[" + province.getString("name") + "][" + city.getString("name") + "]" + county.getString("name") + "]" + town.getString("name") + "]" + village.getString("name") + "]");
                                                    villageList.add(village);
                                                }
                                            }

                                            town.put("children", villageList);
                                            townList.add(town);
                                        }
                                    }

                                    county.put("children", townList);
                                    countyList.add(county);
                                }
                            }
                            city.put("children", countyList);
                            cityList.add(city);
                        }
                    }
                    province.put("children", cityList);
                    provinceList.add(province);
                }
            }
        }
        return provinceList;
    }

    public static String getHtml(String url) {
        return HttpUtil.get(url);
    }
}
